package spider.common;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;

import nlp.model.Message;

/**
 * 存放爬虫使用公共结构
 * 要求：
 * 具有检测是否已爬过该id的功能
 * @author coder
 *
 */
public class CommonDS {
	
	private static HashSet<Long> crawlList;//存放待爬id
	private static HashSet<Long> finishList;//存放已爬完的id
	private static ArrayList<Message> crawlMessageList;//存放已经抓取下的微波数据
	
	
	public static void pullCrawlId(long sinaId){
		if(crawlList==null){
			crawlList=new HashSet<Long>();
		}
		crawlList.add(sinaId);
	}
	
	public static long getCrawlId(){
		long crawlId=0;
		if(crawlList==null || crawlList.size()==0){
			return 0;
		}else{
			Iterator<Long> it=crawlList.iterator();
			if(it.hasNext()){
				crawlId=it.next();
				it.remove();
			}
		}
		return crawlId;
	}
	
	
	
	public static void main(String[] args){
		for(int i =0 ;i < 100;i++){
			pullCrawlId(i);
		}
		for(int i =0 ; i < 100 ; i++){
			System.out.println(getCrawlId()+" ---  "+crawlList.size());
			
		}
	}

}
